package com.jlg.util;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.html.dom.HTMLDocumentImpl;
import org.apache.xerces.xni.parser.XMLDocumentFilter;
import org.cyberneko.html.parsers.DOMFragmentParser;
import org.cyberneko.html.parsers.DOMParser;
import org.w3c.dom.Document;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import com.jlg.beans.page.Content;
import com.jlg.beans.page.Metadata;
import com.jlg.beans.page.Outlink;
import com.jlg.util.dom.DOMContentUtils;
import com.jlg.util.dom.HTMLMetaProcessor;
import com.jlg.util.dom.HTMLMetaTags;


/**
 *
 * author: huazhang 
 * since : 2011-3-26
 */
public final class HttpUtil
{
    // HTTP HEADER
    public final static String CONTENT_TYPE = "Content-Type";
    public final static String CONTENT_LENGTH = "Content-Length";
    public final static String LAST_MODIFIED = "Last-Modified";
    public final static String DATE = "Date";
    
    public static final String FETCH_TIME = "fetch-time";
    
    public static EncodingDetectUtil encodingDetectUtil = new EncodingDetectUtil();
    public static final Log LOG = LogFactory.getLog(HttpUtil.class);
    
    // I used 1000 bytes at first, but  found that some documents have 
    // meta tag well past the first 1000 bytes. 
    // (e.g. http://cn.promo.yahoo.com/customcare/music.html)
    private static final int CHUNK_SIZE = 2000;
    private static Pattern metaPattern = Pattern.compile("<meta\\s+([^>]*http-equiv=\"?content-type\"?[^>]*)>", Pattern.CASE_INSENSITIVE);
    private static Pattern charsetPattern = Pattern.compile("charset=\\s*([a-z][_\\-0-9a-z]*)", Pattern.CASE_INSENSITIVE);
    public static XPath xpath = XPathFactory.newInstance().newXPath();
    private static DOMContentUtils domContentUtils = new DOMContentUtils();
   
    

    public static void main(String[] args) throws Exception{
        
    }
    
    /* 通过正则过滤掉命名空间，以免构建DOM树时出现问题 */
    public byte[] delNameSpace(byte[] content, String encoding) throws Exception
    {
        String regex = "<\\??/?\\w+:\\w+\\s*[^>]{0,}>";
        String temp = new String(content, encoding);
        Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
        Matcher matcher = pattern.matcher(temp);
        while (matcher.find())
        {
            String group = matcher.group();
            if (matcher.start() != 0)
            {
                if (matcher.end() == temp.length())
                {
                    temp = temp.substring(0, matcher.start()) + group.replaceFirst(":", "");
                } else
                {
                    temp = temp.substring(0, matcher.start()) + group.replaceFirst(":", "") + temp.substring(matcher.end());
                }
            } else
            {
                if (matcher.end() == temp.length())
                {
                    temp = group.replaceFirst(":", "");
                } else
                {
                    temp = group.replaceFirst(":", "") + temp.substring(matcher.end());
                }
            }
            matcher = pattern.matcher(temp);
        }

        regex = "<[^>]{1,}\\s+[\\w\\-]+:(=|\\w+)[^>]{0,}>";
        pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
        matcher = pattern.matcher(temp);
        while (matcher.find())
        {
            String group = matcher.group();
            Pattern pt2 = Pattern.compile("\\s+[\\w\\-]+:(=|\\w+)", Pattern.CASE_INSENSITIVE);
            Matcher matcher2 = pt2.matcher(group);
            while (matcher2.find())
            {
                if (matcher2.start() != 0)
                {
                    if (matcher2.end() == group.length())
                    {
                        group = group.substring(0, matcher2.start()) + matcher2.group().replaceFirst(":", "");
                    } else
                    {
                        group = group.substring(0, matcher2.start()) + matcher2.group().replaceFirst(":", "") + group.substring(matcher2.end());
                    }
                } else
                {
                    if (matcher2.end() == group.length())
                    {
                        group = matcher2.group().replaceFirst(":", "");
                    } else
                    {
                        group = matcher2.group().replaceFirst(":", "") + group.substring(matcher2.end());
                    }
                }
                matcher2 = pt2.matcher(group);
            }
            if (matcher.start() != 0)
            {
                if (matcher.end() == temp.length())
                {
                    temp = temp.substring(0, matcher.start()) + group;
                } else
                {
                    temp = temp.substring(0, matcher.start()) + group + temp.substring(matcher.end());
                }
            } else
            {
                if (matcher.end() == temp.length())
                {
                    temp = group;
                } else
                {
                    temp = group + temp.substring(matcher.end());
                }
            }
            matcher = pattern.matcher(temp);
        }
        return temp.getBytes(encoding);
    }
    
    
    public static String getUrl(URL action){
        if (action == null)
        {
            return null;
        }
        StringBuffer urlSb = new StringBuffer(action.getProtocol() + "://" + action.getHost());
        if (action.getPort() != -1)
        {
            urlSb.append(":" + action.getPort());
        }
        if (action.getPath() != null)
        {
            urlSb.append(action.getPath());
        }
        if (action.getQuery() != null)
        {
            urlSb.append("?" + action.getQuery());
        }
        if (action.getRef() != null)
        {
            urlSb.append("#" + action.getRef());
        }
        return urlSb.toString();
    }
    
    public Document createDom(Content content, String encoding) throws Exception
    {
        return createDom(content.getContent(), encoding, null);
    }
    
    public Document createDom(Content content, String encoding, XMLDocumentFilter[] filters) throws Exception
    {
        return createDom(content.getContent(), encoding, filters);
    }

    public Document createDom(byte[] content, String encoding, XMLDocumentFilter[] filters) throws Exception
    {
        InputStreamReader reader = null;
        if (encoding != null)
        {
            reader = new InputStreamReader(new ByteArrayInputStream(content), encoding);
        }
        else
        {
            reader = new InputStreamReader(new ByteArrayInputStream(content));
        }
        InputSource input = new InputSource(reader);
        return createDom(input, encoding, filters);
    }

    public Document createDom(InputSource input, String defaultEncoding, XMLDocumentFilter[] filters) throws Exception
    {
        return createNekoDom(input, defaultEncoding, filters);
    }
    
    private Document createNekoDom(InputSource input, String defaultEncoding, XMLDocumentFilter[] filters) throws Exception
    {
        DOMParser parser = new DOMParser();
        try
        {
            if (defaultEncoding == null || "".equals(defaultEncoding.trim()))
            {
                defaultEncoding = "GBK";
            }
            // 如果整理识别出的元素名称 upper,lower,match
            parser.setProperty( "http://cyberneko.org/html/properties/names/elems", "lower");
            // 如果整理识别出的属性名称 upper,lower,no-change
            parser.setProperty("http://cyberneko.org/html/properties/names/attrs", "lower");
            // 是否允许增补缺失的标签。如果要以XML方式操作HTML文件，此值必须为真。此处提供设置功能，为了性能的原因。
            parser.setFeature("http://cyberneko.org/html/features/balance-tags", true);
            // 是否忽略文档根元素以后的数据。如果为false，<html>和<bod>被忽略，所有的内容都被解析。
            parser.setFeature("http://cyberneko.org/html/features/balance-tags/ignore-outside-content",true);
            // 解析HTML片段时是否作标签增补。此功能不要用在DOMParser上，而要用在DOMFragmentParser上。
            parser.setFeature("http://cyberneko.org/html/features/balance-tags/document-fragment",true);
            // 当遇到字符实体引用（如＆#x20;）是否将(#x20)报告给相应地文档处理器。
            parser.setFeature("http://apache.org/xml/features/scanner/notify-char-refs",false);
            // 是否剥掉<script>元素中的<!-- -->等注释符
            parser.setFeature("http://cyberneko.org/html/features/scanner/script/strip-comment-delims",true);
            // 默认编码
            parser.setProperty("http://cyberneko.org/html/properties/default-encoding", defaultEncoding);
            parser.setFeature("http://xml.org/sax/features/namespaces", false);
            parser.setFeature("http://cyberneko.org/html/features/override-namespaces", true);
            parser.setFeature("http://cyberneko.org/html/features/augmentations", true);
            // 下面如果设成true,正常情况下都报错：java.lang.AbstractMethodError:
            // org.cyberneko.html.HTMLScanner.getCharacterOffset()
            parser.setFeature("http://cyberneko.org/html/features/report-errors", false);
            // Nekohtml不支持名空间 http://nekohtml.sourceforge.net/faq.html
            if (filters != null)
            {
                parser.setProperty("http://cyberneko.org/html/properties/filters", filters);
            }
        } catch (SAXException e)
        {
            e.printStackTrace();
        }
        parser.parse(input);
        return parser.getDocument();
    }
  
    public static Metadata GetHeader(String sUrl) throws Exception
    {
        Content content = curl("GET", sUrl, null, null, 5000, true, null, null, null);
        if (content == null)
        {
            return new Metadata();
        }
        return content.getMetadata();
    }
    
    public static Content GetContent(String sUrl, boolean isAjaxPage) throws Exception
    {
        return GetContent(sUrl, null, isAjaxPage);
    }
    
    public static Content GetContent(String sUrl, String pageEncoding, boolean isAjaxPage) throws Exception
    {
        if (isAjaxPage)
        {
            return CorbaUtil.GetContentWithAjax(sUrl, pageEncoding);
        } else
        {
            return curl("GET", sUrl, null, null, 5000, false, null, null,pageEncoding);
        }
    }
    
    public static Content curl(String method, String sUrl, Map<String, String> paramMap, Map<String, String> requestHeaderMap, int timeout, boolean isOnlyReturnHeader, String basicAuthUser, String basicAuthPass, String pageEncoding)
    {
        if(timeout <= 0){
            timeout = 5000;
        }
        Content content = null;
        HttpURLConnection httpUrlConnection = null;
        InputStream in = null; 
        try
        {     
            sUrl = Utils.encodeUrl(sUrl);
            URL url = new URL(sUrl);
            
            boolean isPost = "POST".equals(method);
            if (Utils.isEmpty(method) || (!"GET".equalsIgnoreCase(method) && !"POST".equalsIgnoreCase(method)))
            {
                method = "POST";
            }
            URL resolvedURL = url;
            if ("GET".equals(method) && !Utils.isEmpty(paramMap))
            {
                boolean firstParam = true;
                StringBuffer newUrlBuffer = new StringBuffer(url.toExternalForm());
                if (url.getQuery() == null)
                {
                    newUrlBuffer.append("?");
                } else
                {
                    newUrlBuffer.append("&");
                }
                for (Map.Entry<String, String> entry : paramMap.entrySet())
                {
                    String encName = URLEncoder.encode(entry.getKey(), "UTF-8");
                    if (firstParam)
                    {
                        firstParam = false;
                    } else
                    {
                        newUrlBuffer.append("&");
                    }
                    String encValue = URLEncoder.encode(entry.getValue(),"UTF-8");
                    newUrlBuffer.append(encName);
                    newUrlBuffer.append("=");
                    newUrlBuffer.append(encValue);
                }
                resolvedURL = new java.net.URL(newUrlBuffer.toString());
            }
            if (resolvedURL.getProtocol().equals("file"))
            {
                try
                {
                    // Remove query so it works.
                    String ref = url.getRef();
                    String refText = ref == null || ref.length() == 0 ? "" : "#" + ref;
                    resolvedURL = new URL(resolvedURL.getProtocol(), url.getHost(), url.getPort(), url.getPath() + refText);
                } catch (java.net.MalformedURLException throwable)
                {
                    resolvedURL = url;
                }
            }

            URLConnection urlConnection = url.openConnection();
            httpUrlConnection = (HttpURLConnection) urlConnection;
            httpUrlConnection.setRequestMethod(method);
            // Do not follow redirects, We will handle redirects ourself
            httpUrlConnection.setInstanceFollowRedirects(false);
            urlConnection.setDoOutput(true);
            urlConnection.setDoInput(true);
            urlConnection.setConnectTimeout(timeout);
            urlConnection.setReadTimeout(timeout);
            urlConnection.setUseCaches(false);
            urlConnection.setDefaultUseCaches(false);
            if (basicAuthUser != null && basicAuthPass != null)
            {
                sun.misc.BASE64Encoder base64 = new sun.misc.BASE64Encoder();
                String _basicAuth = "Basic " + base64.encode((basicAuthUser + ":" + basicAuthPass).getBytes());
                httpUrlConnection.setRequestProperty("Authorization", _basicAuth);
            }
            // set request header
            if (!Utils.isEmpty(requestHeaderMap))
            {
                for (Map.Entry<String, String> entry : requestHeaderMap.entrySet())
                {
                    String key = entry.getKey();
                    String val = entry.getValue();
                    if (key != null && val != null)
                    {
                        urlConnection.setRequestProperty(key, val);
                    }
                }
            } 
            if (isPost)
            {
                urlConnection.setDoOutput(true);
                ByteArrayOutputStream bufOut = new ByteArrayOutputStream();
                boolean firstParam = true;
                for (Map.Entry<String, String> entry : requestHeaderMap.entrySet())
                {
                    String encName = URLEncoder.encode(entry.getKey(), "UTF-8");
                    if (firstParam)
                    {
                        firstParam = false;
                    } else
                    {
                        bufOut.write((byte) '&');
                    }
                    String encValue = URLEncoder.encode(entry.getValue(),"UTF-8");
                    bufOut.write(encName.getBytes("UTF-8"));
                    bufOut.write((byte) '=');
                    bufOut.write(encValue.getBytes("UTF-8"));
                }
                byte[] postContent = bufOut.toByteArray();
                if (urlConnection instanceof HttpURLConnection)
                {
                    ((HttpURLConnection) urlConnection).setFixedLengthStreamingMode(postContent.length);
                }
                OutputStream postOut = urlConnection.getOutputStream();
                postOut.write(postContent);
                postOut.flush();
                postOut.close();
            }        
            int responseCode = httpUrlConnection.getResponseCode();  
            // We handle redirects ourself
            if (responseCode == HttpURLConnection.HTTP_MOVED_PERM || responseCode == HttpURLConnection.HTTP_MOVED_TEMP)
            {
                String location = httpUrlConnection.getHeaderField("Location");
                URL newAction = new URL(url, location);
                // Recurse
                StringBuffer newUrlSb = new StringBuffer(newAction.getProtocol() + "://" + newAction.getHost());
                if (newAction.getPort() != -1){
                    newUrlSb.append(":" + newAction.getPort());
                }
                if (newAction.getPath() != null){
                    newUrlSb.append(newAction.getPath());
                }
                if (newAction.getQuery() != null){
                    newUrlSb.append("?" + newAction.getQuery());
                }
                if (newAction.getRef() != null){
                    newUrlSb.append("#" + newAction.getRef());
                }
                return curl("GET", newUrlSb.toString(), null, null, timeout, isOnlyReturnHeader, null, null, pageEncoding);
            } else if (responseCode == HttpURLConnection.HTTP_OK || responseCode == HttpURLConnection.HTTP_CREATED)
            {
                byte[] bytes = new byte[0];
                String encoding = null;
                if (!isOnlyReturnHeader)
                {
                    in = httpUrlConnection.getInputStream();
                    ByteArrayOutputStream bout = new ByteArrayOutputStream();
                    byte[] buf = new byte[1024];
                    while (true)
                    {
                        int rc = in.read(buf);
                        if (rc <= 0)
                        {
                            break;
                        }
                        else
                        {
                            bout.write(buf, 0, rc);
                        }
                    }
                    bytes = bout.toByteArray();
                    encoding = pageEncoding;
                    if (Utils.isEmpty(encoding))
                    {
                        encoding = getEncoding(bytes, httpUrlConnection.getContentType());
                    }     
                    in.close();
                }
                content = new Content(sUrl, bytes);  
                
                content.setMetadata(new Metadata());   
                // only fetch Content-Length and Last-Modified header
                Map<String, List<String>> responseHeaderMap = httpUrlConnection.getHeaderFields();
                for (Map.Entry<String, List<String>> entry : responseHeaderMap.entrySet())
                {
                    String key = entry.getKey();
                    if (!Utils.isEmpty(key) && (CONTENT_LENGTH.equals(key) || LAST_MODIFIED.equals(key) || CONTENT_TYPE.equals(key) || DATE.equals(key)))
                    {
                        List<String> valList = entry.getValue();
                        String str = valList.get(0);
                        if (valList != null && valList.size() == 1 && !Utils.isEmpty(str))
                        {
                            if(LAST_MODIFIED.equals(key)){
                                str = ((Long)(new Date(str).getTime())).toString();
                            }    
                            if(DATE.equals(key)){
                                str = ((Long)(new Date(str).getTime())).toString();
                            }    
                            if(CONTENT_TYPE.equals(key)){
                                if(Utils.isEmpty(encoding)){
                                   encoding = HttpUtil.getEncodingFromContentType( str);
                                }
                                if(!Utils.isEmpty(encoding)){
                                    content.setEncoding(encoding);
                                }
                            }
                            content.getMetadata().add(key, str);
                        }
                    }
                }   
            }
        } catch (Exception e)
        {
            Utils.ERR(e);
            return null;
        } finally
        {
            if (httpUrlConnection != null)
            {
                httpUrlConnection.disconnect();
            }
        }
        content.getMetadata().add(FETCH_TIME, String.valueOf(System.currentTimeMillis()));
        return content;
    }

    public static String getEncodingFromContentType(String contentType){
        String encoding = null;
        if(Utils.isEmpty(contentType)){
            return null;
        }
        StringTokenizer tok = new StringTokenizer(contentType, ";");
        if (tok.hasMoreTokens())
        {
            tok.nextToken();
            while (tok.hasMoreTokens())
            {
                String assignment = tok.nextToken().trim();
                int eqIdx = assignment.indexOf('=');
                if (eqIdx != -1)
                {
                    String varName = assignment.substring(0, eqIdx).trim();
                    if ("charset".equalsIgnoreCase(varName))
                    {
                        String varValue = assignment.substring(eqIdx + 1).trim();
                        if (varValue.startsWith("\"") && varValue.endsWith("\""))
                        {
                            // substring works on indices
                            varValue = varValue.substring(1, varValue.length() - 1);
                        }
                        if (Charset.isSupported(varValue))
                        {
                            encoding = varValue;
                        }
                    }
                }
            }
        }
        return encoding;
    }
    
    public static String sniffCharacterEncoding(byte[] content)
    {
        int length = content.length < CHUNK_SIZE ? content.length : CHUNK_SIZE;
        // We don't care about non-ASCII parts so that it's sufficient
        // to just inflate each byte to a 16-bit value by padding.
        // For instance, the sequence {0x41, 0x82, 0xb7} will be turned into
        // {U+0041, U+0082, U+00B7}.
        String str = "";
        try
        {
            str = new String(content, 0, length, Charset.forName("ASCII").toString());
        } catch (UnsupportedEncodingException e)
        {
            // code should never come here, but just in case...
            return null;
        }

        Matcher metaMatcher = metaPattern.matcher(str);
        String encoding = null;
        if (metaMatcher.find())
        {
            Matcher charsetMatcher = charsetPattern.matcher(metaMatcher.group(1));
            if (charsetMatcher.find()){
                String varValue = new String(charsetMatcher.group(1));
                if (Charset.isSupported(varValue))
                {
                    encoding = varValue;
                }
            }       
        }
        return encoding;
    }
    
    public static String getEncoding(byte[] bytes, String contentType)
    {
        String encoding = null;
        // 分析HTTP头获取编码
        if (!Utils.isEmpty(contentType))
        {
            encoding = getEncodingFromContentType(contentType);
        }
        // 分析网页文本获取编码
        if (Utils.isEmpty(encoding))
        {
            encoding = sniffCharacterEncoding(bytes);
        }
        // 分析网页字节流获取编码
        if (Utils.isEmpty(encoding))
        {
            encoding = encodingDetectUtil.detectEncoding(bytes);
        }
        return encoding;
    }
    
    
    public Outlink[] getOutLinkByXpath(String Expression, Object doc, String url) throws Exception
    {
       
        URL turl = new URL(url);
        NodeList nl = (NodeList) xpath.evaluate(Expression, doc, XPathConstants.NODESET);
        String tmp = "";
        Set<Outlink> l = new HashSet<Outlink>();  // 用Set能去重
        for (int i = 0; i < nl.getLength(); i++)
        {
            if (nl.item(i).getNodeValue() != null)
                tmp = nl.item(i).getNodeValue().trim();
            // 过滤这种链接，D:\1751565968.jpg
            if (!"".equals(tmp) && tmp.length() > 2 && !tmp.substring(1, 2).equals(":"))
            {
                tmp = tmp.replaceAll("\\\\", "/");
                URL nurl = new URL(turl, tmp);
                Outlink ol = new Outlink(nurl.toString(), null);
                l.add(ol);
            }
        }
        return l.toArray(new Outlink[l.size()]);
    }

    public Outlink[] getOutLink(Content content) throws Exception
    {
        return getOutLink(content, null, null);
    }

    public Outlink[] getOutLink(Content content, String regex, String encoding) throws Exception
    {
        try
        {
            byte contentInOctets[] = content.getContent();
            InputSource input = new InputSource(new ByteArrayInputStream( contentInOctets));
            if (Utils.isEmpty(encoding))
            {
                encoding = getEncoding(contentInOctets, null);
            }
            input.setEncoding(encoding);
            DOMFragmentParser parser = new DOMFragmentParser();
            try
            {
                parser.setProperty("http://cyberneko.org/html/properties/names/elems","lower");
                // parser.setFeature("http://xml.org/sax/features/namespaces ",false);
                parser.setFeature("http://cyberneko.org/html/features/override-doctype", true);
                parser.setFeature("http://cyberneko.org/html/features/override-namespaces", true);
                parser.setFeature("http://cyberneko.org/html/features/balance-tags/ignore-outside-content", true);
                parser.setFeature("http://cyberneko.org/html/features/scanner/script/strip-comment-delims", true);
                parser.setFeature("http://cyberneko.org/html/features/scanner/script/strip-cdata-delims", true);
                parser.setFeature( "http://cyberneko.org/html/features/scanner/style/strip-comment-delims", true);
                parser.setFeature("http://cyberneko.org/html/features/augmentations",true);
                parser.setFeature( "http://cyberneko.org/html/features/scanner/style/strip-comment-delims",true);
                parser.setFeature("http://cyberneko.org/html/features/balance-tags/document-fragment",true);
                parser.setFeature( "http://cyberneko.org/html/features/report-errors", false);
            } catch (SAXException e)
            {
                e.printStackTrace();
            }
            HTMLDocumentImpl doc = new HTMLDocumentImpl();
            doc.setErrorChecking(false);
            DocumentFragment fragment = doc.createDocumentFragment();
            parser.parse(input, fragment);
            
            Outlink outlinks[] = new Outlink[0];

            HTMLMetaTags metaTags = new HTMLMetaTags();
            URL base = new URL(content.getUrl());
            HTMLMetaProcessor.getMetaTags(metaTags, fragment, base);

            if (!metaTags.getNoFollow())
            {
                ArrayList<Outlink> l = new ArrayList<Outlink>();
                URL baseTag = domContentUtils.getBase(fragment);
                domContentUtils.getOutlinks(baseTag == null ? base : baseTag, l, fragment, regex);
                outlinks = l.toArray(new Outlink[l.size()]);
            }
            return outlinks;
        } catch (Exception e)
        {
            e.printStackTrace();
            throw e;
        }
    }
    
    /** 打印结点* */
    public String print(Node node)
    {
        StringBuffer sb = new StringBuffer();
        sb.setLength(0);
        getText(sb, node);
        return sb.toString();
    }

    /** 取出一个结点的所有内容 **/
    public void getText(StringBuffer sb, Node node)
    {
        if (node == null)
        {
            return;
        }
        int type = node.getNodeType();
        switch (type)
        {
        case Node.DOCUMENT_NODE:
            // 对www.163.com这个首先报错：org.w3c.dom.DOMException:
            // HIERARCHY_REQUEST_ERR
            try
            {
                getText(sb, ((Document) node).getDocumentElement());
            } catch (Exception e)
            {
                sb.append("从Document对象中提取文本发生如下错误：\n" + e.getMessage());
                return;
            }
            break;
        case Node.DOCUMENT_FRAGMENT_NODE:
            NodeList list = node.getChildNodes();
            if (list != null && list.getLength() > 0)
            {
                for (int i = 0; i < list.getLength(); i++)
                {
                    Node eleNode = list.item(i);
                    if (eleNode.getNodeType() == Node.ELEMENT_NODE)
                    {
                        getText(sb, list.item(i));
                    }
                }
            }
            break;
        case Node.ELEMENT_NODE:
            sb.append('<');
            sb.append(node.getNodeName());
            NamedNodeMap attrs = node.getAttributes();
            for (int i = 0; i < attrs.getLength(); i++)
            {
                sb.append(' ');
                sb.append(attrs.item(i).getNodeName());
                sb.append("=\"");
                sb.append(attrs.item(i).getNodeValue());
                sb.append('"');
            }
            sb.append('>');
            sb.append("\n");
            // 输出自定义数据

//            if (node.getUserData("type") != null)
//            {
//                sb.append("<!--\n");
//                sb.append("<blockinfo>\n");
//                sb.append("  <type>" + node.getUserData("type") + "</type>\n");
//                sb.append("  <weight>" + node.getUserData("weight")
//                        + "</weight>\n");
//                sb.append("  <location>" + node.getUserData("location")
//                        + "</location>\n");
//                sb.append("  <width>" + node.getUserData("width")
//                        + "</width>\n");
//                sb.append("  <aligh>" + node.getUserData("aligh")
//                        + "</aligh>\n");
//                sb.append("  <wordSum>" + node.getUserData("wordSum")
//                        + "</wordSum>\n");
//                sb.append("  <picSum>" + node.getUserData("picSum")
//                        + "</picSum>\n");
//                sb.append("  <linkSum>" + node.getUserData("linkSum")
//                        + "</linkSum>\n");
//                sb.append("  <linkList>\n");
//                sb.append(node.getUserData("linkList"));
//                sb.append("  </linkList>\n");
//                sb.append("  <weightyTagList>\n");
//                sb.append(node.getUserData("weightyTagList"));
//                sb.append("  </weightyTagList>\n");
//                sb.append("</blockinfo>\n");
//                sb.append("-->\n");
//            }
            // 递归输出访元素子节点
            NodeList children = node.getChildNodes();
            if (children != null)
            {
                int len = children.getLength();
                for (int i = 0; i < len; i++)
                {
                    getText(sb, children.item(i));
                }
            }
            break;
        case Node.TEXT_NODE:
            sb.append(node.getNodeValue());
            break;

        }
        if (type == Node.ELEMENT_NODE)
        {
            sb.append("</");
            sb.append(node.getNodeName());
            sb.append('>');
            sb.append("\n");
        }
    }
    
    public String getHtmlByXpath(String Expression, Object doc) throws Exception
    {
        XPath xpath = XPathFactory.newInstance().newXPath();
        Node nl = (Node) xpath.evaluate(Expression, doc, XPathConstants.NODE);
        DOMSource source = new DOMSource(nl);
        TransformerFactory transFactory = TransformerFactory.newInstance();
        Transformer transformer;
        try
        {
            transformer = transFactory.newTransformer();
            transformer.setOutputProperty(OutputKeys.METHOD, "html");
            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
            transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
            // //<?xml:namespace prefix = o ns =
            // "urn:schemas-microsoft-com:office:office" />
            // //<?xml:namespace prefix = st1 ns =
            // "urn:schemas-microsoft-com:office:smarttags" />
            // transformer.setParameter("o",
            // "urn:schemas-microsoft-com:office:office");
            // transformer.setParameter("w",
            // "urn:schemas-microsoft-com:office:smarttags");
            StringWriter bs = new StringWriter();
            StreamResult result = new StreamResult(bs);
            transformer.transform(source, result);
            bs.flush();
            return bs.toString();
        } catch (Exception e2)
        {
            e2.printStackTrace();
        }
        return null;
    }

    public String getTextByXptah(String Expression, Object doc) throws Exception
    {
        XPath xpath = XPathFactory.newInstance().newXPath();
        NodeList nl = (NodeList) xpath.evaluate(Expression, doc, XPathConstants.NODESET);
        String tmp = "";
        String title = "";
        for (int i = 0; i < nl.getLength(); i++)
        {
            if (nl.item(i) != null && nl.item(i).getNodeValue() != null)
            {
                tmp = nl.item(i).getNodeValue().trim();
                if (!"".equals(tmp))
                {
                    title += tmp;
                    if (i != (nl.getLength() - 1))
                    {
                        title += "\n";
                    }
                }
            }
        }
        title = title.trim();
        return title;
    }

    public String getResouceSuffix(String url)
    {
        String suffix = "";
        if (url != null && url.length() > 0)
        {
            int pos = -1;
            pos = url.lastIndexOf(".");
            suffix = url.substring(pos + 1);
        }
        return suffix.toLowerCase();
    }

    public void removeNodeByTagNameFromOneNode(Node node, String tagName, String condition)
    {
        List<Node> allNeedToDelNodeList = new ArrayList<Node>();
        getNeedToDelNode(allNeedToDelNodeList, node, tagName, condition);
        if (allNeedToDelNodeList.size() > 0)
        {
            for (int i = 0; i < allNeedToDelNodeList.size(); i++)
            {
                removeOneNode(allNeedToDelNodeList.get(i));
            }
        }
    }

    private void getNeedToDelNode(List<Node> allNeedToDelNodeList, Node node, String tagName, String condition)
    {
        if (node == null || tagName == null)
            return;
        if (node.getNodeType() == Node.ELEMENT_NODE)
        {
            if (tagName.equalsIgnoreCase(node.getNodeName()))
            {
                if (condition == null)
                {
                    allNeedToDelNodeList.add(node);
                    return;
                } else
                {
                    boolean needToDel = true;
                    String[] attrs = condition.split(";");
                    if (attrs != null && attrs.length > 0)
                    {
                        for (String attr : attrs)
                        {
                            if (attr != null)
                            {
                                String[] temp = attr.split("=");
                                if (temp != null && temp.length == 2)
                                {
                                    String attrName = temp[0]; 
                                    String attrVal = temp[1]; 
                                    NamedNodeMap atrrMap = node.getAttributes();
                                    if (atrrMap != null)
                                    {
                                        Node atrrNode = atrrMap.getNamedItem(attrName);
                                        if (atrrNode != null)
                                        {
                                            if (!attrVal.equalsIgnoreCase(atrrNode.getNodeValue()))
                                            {
                                                needToDel = false;
                                                break;
                                            }
                                        }
                                    }
                                }
                            }
                        }// end for
                    }
                    if (needToDel)
                    {
                        allNeedToDelNodeList.add(node);
                        return;
                    } else
                    {
                        return;
                    }
                }// end else
            }
        }
        NodeList children = node.getChildNodes();
        if (children != null)
        {
            int len = children.getLength();
            for (int i = 0; i < len; i++)
            {
                getNeedToDelNode(allNeedToDelNodeList, children.item(i), tagName, condition);
            }
        }
    }

    private void removeOneNode(Node node)
    {
        Node paraNode = node.getParentNode();
        if (paraNode != null)
        {
            paraNode.removeChild(node);
        }
    }

    public String getResouceName(String url)
    {
        String suffix = "";
        if (url != null && url.length() > 0)
        {
            url = url.replaceAll("\\\\", "/");
            url = url.replaceAll("\\?", "/");
            int pos = -1;
            pos = url.lastIndexOf("/");
            suffix = url.substring(pos + 1);
        }
        return suffix.toLowerCase();
    }

    public static byte[] getBytes(InputStream is) throws Exception
    {
        byte[] data = null;
        Collection chunks = new ArrayList();
        byte[] buffer = new byte[1024 * 1000];
        int read = -1;
        int size = 0;
        while ((read = is.read(buffer)) != -1)
        {
            if (read > 0)
            {
                byte[] chunk = new byte[read];
                System.arraycopy(buffer, 0, chunk, 0, read);
                chunks.add(chunk);
                size += chunk.length;
            }
        }
        if (size > 0)
        {
            ByteArrayOutputStream bos = null;
            try
            {
                bos = new ByteArrayOutputStream(size);
                for (Iterator itr = chunks.iterator(); itr.hasNext();)
                {
                    byte[] chunk = (byte[]) itr.next();
                    bos.write(chunk);
                }
                data = bos.toByteArray();
            } finally
            {
                if (bos != null)
                {
                    bos.close();
                }
            }
        }
        return data;
    }
   
}
